notebooks/5.2 Visualize, Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5.2 Visualize/Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we use `dask`'s `DataFrame` to further process and analyse the output of `passim` ([GitHub repo](https://github.com/dasmiq/passim)), a an open source software to detect text reuse developed by David Smith (Northeastern University). \n",
    "\n",
    "The data we use here was produced by running a subset of the [Impresso](http://impresso-project.ch)'s newspaper corpus through `passim`. For furter information and context, see [this slide deck](https://infoscience.epfl.ch/record/255933/files/slides.pdf).\n",
    "\n",
    "**NB**: this notebook **won't work** in binder as it requires input data that are stored in a private s3 repository."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import dask\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from dask import dataframe as dd\n",
    "from dask.distributed import Client\n",
    "from impresso_commons.path.path_s3 import IMPRESSO_STORAGEOPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.1.5'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dask.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read in passim data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# client = cluster.make_dask_client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = Client('localhost:8686')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Client</h3>\n",
       "<ul>\n",
       "  <li><b>Scheduler: </b>tcp://localhost:8686\n",
       "  <li><b>Dashboard: </b><a href='http://localhost:41267/status' target='_blank'>http://localhost:41267/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Cluster</h3>\n",
       "<ul>\n",
       "  <li><b>Workers: </b>48</li>\n",
       "  <li><b>Cores: </b>48</li>\n",
       "  <li><b>Memory: </b>144.00 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://10.90.38.12:8686' processes=48 cores=48>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_bucket = \"s3://processed-canonical-data/text-reuse/\"\n",
    "path_to_data = \"test/out.json/\"\n",
    "path_output = \"test/clusters.json/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = dd.read_json(\n",
    "    f\"{os.path.join(input_bucket, path_to_data)}*.json\", \n",
    "    storage_options=IMPRESSO_STORAGEOPT\n",
    ").set_index('uid').persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>begin</th>\n",
       "      <th>cc</th>\n",
       "      <th>cluster</th>\n",
       "      <th>date</th>\n",
       "      <th>end</th>\n",
       "      <th>gid</th>\n",
       "      <th>id</th>\n",
       "      <th>pages</th>\n",
       "      <th>series</th>\n",
       "      <th>size</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>uid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-9223360541113828267</th>\n",
       "      <td>168</td>\n",
       "      <td>True</td>\n",
       "      <td>352187320617</td>\n",
       "      <td>1968-02-24</td>\n",
       "      <td>955</td>\n",
       "      <td>2328324961100034523</td>\n",
       "      <td>GDL-1968-02-24-a-i0240</td>\n",
       "      <td>[{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...</td>\n",
       "      <td>GDL</td>\n",
       "      <td>10</td>\n",
       "      <td>TV.\\n18.45 Bulletin de nouvelles.\\n18.50 Publi...</td>\n",
       "      <td>• Tél é ^tr e bon plaisir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-9223360541113828267</th>\n",
       "      <td>39</td>\n",
       "      <td>True</td>\n",
       "      <td>32</td>\n",
       "      <td>1968-02-24</td>\n",
       "      <td>2017</td>\n",
       "      <td>2328324961100034523</td>\n",
       "      <td>GDL-1968-02-24-a-i0240</td>\n",
       "      <td>[{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...</td>\n",
       "      <td>GDL</td>\n",
       "      <td>80757</td>\n",
       "      <td>SAMEDI 24 FÉVRIER\\nProgramme suisse\\n14. 00-15...</td>\n",
       "      <td>• Tél é ^tr e bon plaisir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-9223360541113828267</th>\n",
       "      <td>2027</td>\n",
       "      <td>True</td>\n",
       "      <td>32</td>\n",
       "      <td>1968-02-24</td>\n",
       "      <td>4301</td>\n",
       "      <td>2328324961100034523</td>\n",
       "      <td>GDL-1968-02-24-a-i0240</td>\n",
       "      <td>[{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...</td>\n",
       "      <td>GDL</td>\n",
       "      <td>80757</td>\n",
       "      <td>DIMANCHE 25 FÉVRIER\\nProgramme suisse\\n10.15 P...</td>\n",
       "      <td>• Tél é ^tr e bon plaisir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-9223360541113828267</th>\n",
       "      <td>3166</td>\n",
       "      <td>True</td>\n",
       "      <td>1228360652012</td>\n",
       "      <td>1968-02-24</td>\n",
       "      <td>3457</td>\n",
       "      <td>2328324961100034523</td>\n",
       "      <td>GDL-1968-02-24-a-i0240</td>\n",
       "      <td>[{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...</td>\n",
       "      <td>GDL</td>\n",
       "      <td>3</td>\n",
       "      <td>17.15 Kiri le Clown.\\n17.25 Ramuntcho (1958), ...</td>\n",
       "      <td>• Tél é ^tr e bon plaisir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-9223360541113828267</th>\n",
       "      <td>2093</td>\n",
       "      <td>True</td>\n",
       "      <td>17179874715</td>\n",
       "      <td>1968-02-24</td>\n",
       "      <td>2223</td>\n",
       "      <td>2328324961100034523</td>\n",
       "      <td>GDL-1968-02-24-a-i0240</td>\n",
       "      <td>[{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...</td>\n",
       "      <td>GDL</td>\n",
       "      <td>55</td>\n",
       "      <td>11.00 Un'ora per voi, reprise.\\n12.00 Table ou...</td>\n",
       "      <td>• Tél é ^tr e bon plaisir</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      begin    cc        cluster       date   end  \\\n",
       "uid                                                                 \n",
       "-9223360541113828267    168  True   352187320617 1968-02-24   955   \n",
       "-9223360541113828267     39  True             32 1968-02-24  2017   \n",
       "-9223360541113828267   2027  True             32 1968-02-24  4301   \n",
       "-9223360541113828267   3166  True  1228360652012 1968-02-24  3457   \n",
       "-9223360541113828267   2093  True    17179874715 1968-02-24  2223   \n",
       "\n",
       "                                      gid                      id  \\\n",
       "uid                                                                 \n",
       "-9223360541113828267  2328324961100034523  GDL-1968-02-24-a-i0240   \n",
       "-9223360541113828267  2328324961100034523  GDL-1968-02-24-a-i0240   \n",
       "-9223360541113828267  2328324961100034523  GDL-1968-02-24-a-i0240   \n",
       "-9223360541113828267  2328324961100034523  GDL-1968-02-24-a-i0240   \n",
       "-9223360541113828267  2328324961100034523  GDL-1968-02-24-a-i0240   \n",
       "\n",
       "                                                                  pages  \\\n",
       "uid                                                                       \n",
       "-9223360541113828267  [{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...   \n",
       "-9223360541113828267  [{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...   \n",
       "-9223360541113828267  [{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...   \n",
       "-9223360541113828267  [{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...   \n",
       "-9223360541113828267  [{'id': 'GDL-1968-02-24-a-p0020', 'seq': 20, '...   \n",
       "\n",
       "                     series   size  \\\n",
       "uid                                  \n",
       "-9223360541113828267    GDL     10   \n",
       "-9223360541113828267    GDL  80757   \n",
       "-9223360541113828267    GDL  80757   \n",
       "-9223360541113828267    GDL      3   \n",
       "-9223360541113828267    GDL     55   \n",
       "\n",
       "                                                                   text  \\\n",
       "uid                                                                       \n",
       "-9223360541113828267  TV.\\n18.45 Bulletin de nouvelles.\\n18.50 Publi...   \n",
       "-9223360541113828267  SAMEDI 24 FÉVRIER\\nProgramme suisse\\n14. 00-15...   \n",
       "-9223360541113828267  DIMANCHE 25 FÉVRIER\\nProgramme suisse\\n10.15 P...   \n",
       "-9223360541113828267  17.15 Kiri le Clown.\\n17.25 Ramuntcho (1958), ...   \n",
       "-9223360541113828267  11.00 Un'ora per voi, reprise.\\n12.00 Table ou...   \n",
       "\n",
       "                                          title  \n",
       "uid                                              \n",
       "-9223360541113828267  • Tél é ^tr e bon plaisir  \n",
       "-9223360541113828267  • Tél é ^tr e bon plaisir  \n",
       "-9223360541113828267  • Tél é ^tr e bon plaisir  \n",
       "-9223360541113828267  • Tél é ^tr e bon plaisir  \n",
       "-9223360541113828267  • Tél é ^tr e bon plaisir  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_rows, n_cols = df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1170211"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_rows.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters_df = df.groupby('cluster').agg({'date': ['min', 'max'], 'size': 'count'}).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters_df.loc[:,'time_delta'] = clusters_df.date['max'] - clusters_df.date['min']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">date</th>\n",
       "      <th>size</th>\n",
       "      <th>time_delta</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>count</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cluster</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1988-11-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2313 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              date             size time_delta\n",
       "               min        max count           \n",
       "cluster                                       \n",
       "0       1982-07-24 1982-07-24     2     0 days\n",
       "1       1982-07-24 1988-11-22     2  2313 days\n",
       "2       1972-02-17 1972-02-17     2     0 days\n",
       "3       1972-02-17 1972-02-17     2     0 days\n",
       "4       1972-02-17 1972-02-17     2     0 days"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cluster\n",
       "32               80757\n",
       "89                2649\n",
       "267                 65\n",
       "279                 53\n",
       "283                 35\n",
       "287                 34\n",
       "532               3256\n",
       "674                642\n",
       "821                152\n",
       "3301               243\n",
       "4326                 2\n",
       "4327                 2\n",
       "4328                 2\n",
       "4329                 2\n",
       "4330                 2\n",
       "4331                 2\n",
       "8589936556          35\n",
       "8589938473          32\n",
       "8589938915           2\n",
       "8589938943           2\n",
       "8589938944           2\n",
       "8589939157          45\n",
       "8589939175          71\n",
       "8589940471         302\n",
       "17179869184          3\n",
       "17179869786         25\n",
       "17179871344          2\n",
       "17179873216          2\n",
       "17179873350         27\n",
       "17179874715         55\n",
       "                 ...  \n",
       "1159641178619        2\n",
       "1271310322041        2\n",
       "17179873390          2\n",
       "17179873391          2\n",
       "17179875608          2\n",
       "34359742823          2\n",
       "42949675053          2\n",
       "42949679694          2\n",
       "94489286978          2\n",
       "94489286979          2\n",
       "94489289113          2\n",
       "188978564943         2\n",
       "231928240365         2\n",
       "240518174726         2\n",
       "274877908875         2\n",
       "335007457587         2\n",
       "420906801607         2\n",
       "455266535639         2\n",
       "678604839730         2\n",
       "695784710894         2\n",
       "747324316383         2\n",
       "850403528854         2\n",
       "1108101570802        2\n",
       "1133871372599        2\n",
       "1159641173835        2\n",
       "1202590851241        2\n",
       "1202590851244        2\n",
       "120259088382         2\n",
       "455266537802         2\n",
       "678604839738         2\n",
       "Name: size, Length: 458569, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# that's a silly one, just count TR instances per cluster\n",
    "df.groupby('cluster')['size'].count().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the list of newspapers for which a cluster contains TR instances\n",
    "np = df.groupby('cluster').apply(lambda r:\",\".join(sorted(r['series'].unique())), meta=('np', object)).compute().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters_df.loc[:, 'newspapers'] = np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">date</th>\n",
       "      <th>size</th>\n",
       "      <th>time_delta</th>\n",
       "      <th>newspapers</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>count</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cluster</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1988-11-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2313 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              date             size time_delta newspapers\n",
       "               min        max count                      \n",
       "cluster                                                  \n",
       "0       1982-07-24 1982-07-24     2     0 days    GDL,IMP\n",
       "1       1982-07-24 1988-11-22     2  2313 days    GDL,IMP\n",
       "2       1972-02-17 1972-02-17     2     0 days    GDL,IMP\n",
       "3       1972-02-17 1972-02-17     2     0 days    GDL,IMP\n",
       "4       1972-02-17 1972-02-17     2     0 days    GDL,IMP"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mint_document_ids(row):\n",
    "    \n",
    "    ids = row['id']\n",
    "    begins = row['begin']\n",
    "    ends = row['end']\n",
    "    \n",
    "    return \",\".join([\n",
    "        \"{}@{}:{}\".format(doc_id, begin, end)\n",
    "        for doc_id, begin, end in zip(ids, begins, ends)\n",
    "    ])\n",
    "    \n",
    "    return "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_ids = df.groupby('cluster').apply(mint_document_ids, meta=('document_ids', object)).compute().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters_df.loc[:, 'members'] = doc_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">date</th>\n",
       "      <th>size</th>\n",
       "      <th>time_delta</th>\n",
       "      <th>members</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>count</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cluster</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1988-11-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2313 days</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              date             size time_delta  \\\n",
       "               min        max count              \n",
       "cluster                                          \n",
       "0       1982-07-24 1982-07-24     2     0 days   \n",
       "1       1982-07-24 1988-11-22     2  2313 days   \n",
       "2       1972-02-17 1972-02-17     2     0 days   \n",
       "3       1972-02-17 1972-02-17     2     0 days   \n",
       "4       1972-02-17 1972-02-17     2     0 days   \n",
       "\n",
       "                                                   members  \n",
       "                                                            \n",
       "cluster                                                     \n",
       "0        IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...  \n",
       "1        IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...  \n",
       "2        GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...  \n",
       "3        GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...  \n",
       "4        GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "code_folding": [
     3
    ]
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "\n",
    "def lexicaloverlap(row):\n",
    "    texts = row['text']\n",
    "    first = True\n",
    "    intersection = list()\n",
    "    \n",
    "    longest_text_lenght = max([len(text) for text in texts])\n",
    "    \n",
    "    for text in texts:\n",
    "        if(first):\n",
    "            first = False\n",
    "            intersection = re.sub('[().,;:!0-9\"{}\\][»«]','',text).lower().split()\n",
    "        else:\n",
    "            processing = re.sub('[().,;:!0-9\"{}\\][»«]','',text).lower().split()\n",
    "            intersection = np.intersect1d(processing, intersection)\n",
    "            \n",
    "    overlap_pct = (len(intersection) * 100) / longest_text_lenght\n",
    "    return overlap_pct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "\n",
    "def lexicaloverlap_revised(row):\n",
    "    texts = row['text']\n",
    "    first = True\n",
    "    intersection = list()\n",
    "    \n",
    "    longest_text_length = max([len(text) for text in texts])\n",
    "    \n",
    "    token_sets = [\n",
    "        set(re.sub('[().,;:!0-9\"{}\\][»«]','',text).lower().split())\n",
    "        for text in texts\n",
    "    ]\n",
    "    \n",
    "    longest_text_length = max([len(ts) for ts in token_sets])\n",
    "    intersection = set.intersection(*token_sets)\n",
    "    overlap_pct = (len(intersection) * 100) / longest_text_length\n",
    "    return overlap_pct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "overlap = df.groupby('cluster').apply(lexicaloverlap, meta=('lexical_overlap', object)).compute().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "overlap2 = df.groupby('cluster').apply(lexicaloverlap_revised, meta=('lexical_overlap2', object)).compute().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "67.5"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overlap2.median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters_df.loc[:, 'lexical_overlap'] = overlap2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">date</th>\n",
       "      <th>size</th>\n",
       "      <th>time_delta</th>\n",
       "      <th>members</th>\n",
       "      <th>lexical_overlap</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>count</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cluster</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...</td>\n",
       "      <td>76.265823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1988-11-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2313 days</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...</td>\n",
       "      <td>64.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...</td>\n",
       "      <td>82.857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...</td>\n",
       "      <td>82.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...</td>\n",
       "      <td>65.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              date             size time_delta  \\\n",
       "               min        max count              \n",
       "cluster                                          \n",
       "0       1982-07-24 1982-07-24     2     0 days   \n",
       "1       1982-07-24 1988-11-22     2  2313 days   \n",
       "2       1972-02-17 1972-02-17     2     0 days   \n",
       "3       1972-02-17 1972-02-17     2     0 days   \n",
       "4       1972-02-17 1972-02-17     2     0 days   \n",
       "\n",
       "                                                   members lexical_overlap  \n",
       "                                                                            \n",
       "cluster                                                                     \n",
       "0        IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...       76.265823  \n",
       "1        IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...       64.285714  \n",
       "2        GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...       82.857143  \n",
       "3        GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...       82.142857  \n",
       "4        GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...       65.000000  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = df.groupby('cluster').apply(lambda x: x['size'], meta=('cluster_size', object)).compute().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters_df.loc[:, 'cluster_size'] = clusters_df['size']['count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">date</th>\n",
       "      <th>size</th>\n",
       "      <th>time_delta</th>\n",
       "      <th>members</th>\n",
       "      <th>lexical_overlap</th>\n",
       "      <th>cluster_size</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>count</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cluster</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...</td>\n",
       "      <td>76.265823</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1988-11-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2313 days</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...</td>\n",
       "      <td>64.285714</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...</td>\n",
       "      <td>82.857143</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...</td>\n",
       "      <td>82.142857</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              date             size time_delta  \\\n",
       "               min        max count              \n",
       "cluster                                          \n",
       "0       1982-07-24 1982-07-24     2     0 days   \n",
       "1       1982-07-24 1988-11-22     2  2313 days   \n",
       "2       1972-02-17 1972-02-17     2     0 days   \n",
       "3       1972-02-17 1972-02-17     2     0 days   \n",
       "4       1972-02-17 1972-02-17     2     0 days   \n",
       "\n",
       "                                                   members lexical_overlap  \\\n",
       "                                                                             \n",
       "cluster                                                                      \n",
       "0        IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...       76.265823   \n",
       "1        IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...       64.285714   \n",
       "2        GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...       82.857143   \n",
       "3        GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...       82.142857   \n",
       "4        GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...       65.000000   \n",
       "\n",
       "        cluster_size  \n",
       "                      \n",
       "cluster               \n",
       "0                  2  \n",
       "1                  2  \n",
       "2                  2  \n",
       "3                  2  \n",
       "4                  2  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f7b1b00a7b8>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAVgklEQVR4nO3df4xd5X3n8fd37UIcusEGqhFre9dexUrl4M2GjMBVVtUIVzAQFPMHSY3YYFg31irQ0Mqr1HT/sDYJUqJtSkM2QbJi1yZCOKybXVvFqWsBV9n9ww52qDCGZBkZiGdkII2NqYMSdrLf/eM+o95O5rHje2fu9cx9v6SrOed7nnPO83CG+/H5ce9EZiJJ0lT+Wa87IEm6eBkSkqQqQ0KSVGVISJKqDAlJUtX8Xndgul111VW5bNmyttb92c9+xmWXXTa9HbrIOeb+4Jj7QydjPnLkyN9n5m9Nrs+5kFi2bBmHDx9ua91Go8HQ0ND0dugi55j7g2PuD52MOSJem6ru5SZJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklRlSEiSqgwJSVLVnPvEtaSLx7LNT/Zs3zuG++srOWaKZxKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVHXekIiI7RHxZkS80FL7rxHxw4h4PiL+R0QsbFn2QESMRMSPIuKmlvpwqY1ExOaW+vKIOFTq346IS0r90jI/UpYvm65BS5J+Pb/OmcQOYHhS7QBwTWb+G+D/AA8ARMRKYB3wwbLONyJiXkTMA74O3AysBO4obQG+DDyUme8HTgMbSn0DcLrUHyrtJElddN6QyMzvAacm1f42M8fL7EFgSZleC+zKzF9k5ivACHBdeY1k5vHMfBfYBayNiABuAHaX9XcCt7Vsa2eZ3g2sKe0lSV0yHfck/gPw3TK9GDjRsmy01Gr1K4G3WgJnov5PtlWWnyntJUld0tFXhUfEfwbGgcempztt92MjsBFgYGCARqPR1nbOnj3b9rqzlWPuD70a86ZV4+dvNEM8ztOj7ZCIiLuBW4E1mZmlPAYsbWm2pNSo1H8KLIyI+eVsobX9xLZGI2I+cHlp/ysycyuwFWBwcDCHhobaGlOj0aDddWcrx9wfejXmu3v89yQ8zp1r63JTRAwDnwM+npnvtCzaC6wrTyYtB1YA3weeBVaUJ5kuoXlze28Jl2eA28v664E9LdtaX6ZvB55uCSNJUhec90wiIh4HhoCrImIU2ELzaaZLgQPlXvLBzPyPmXksIp4AXqR5GerezPxl2c59wH5gHrA9M4+VXfwJsCsivgg8B2wr9W3AtyJihOaN83XTMF5J0gU4b0hk5h1TlLdNUZto/yDw4BT1fcC+KerHaT79NLn+c+AT5+ufJGnm+IlrSVKVISFJqjIkJElVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklR13pCIiO0R8WZEvNBSuyIiDkTEy+XnolKPiHg4IkYi4vmIuLZlnfWl/csRsb6l/pGIOFrWeTgi4lz7kCR1z69zJrEDGJ5U2ww8lZkrgKfKPMDNwIry2gg8As03fGALcD1wHbCl5U3/EeDTLesNn2cfkqQuOW9IZOb3gFOTymuBnWV6J3BbS/3RbDoILIyIq4GbgAOZeSozTwMHgOGy7H2ZeTAzE3h00ram2ockqUvavScxkJkny/TrwECZXgycaGk3Wmrnqo9OUT/XPiRJXTK/0w1kZkZETkdn2t1HRGykeXmLgYEBGo1GW/s5e/Zs2+vOVo65P/RqzJtWjXd9nxM8ztOj3ZB4IyKuzsyT5ZLRm6U+Bixtabek1MaAoUn1RqkvmaL9ufbxKzJzK7AVYHBwMIeGhmpNz6nRaNDuurOVY+4PvRrz3Zuf7Po+J+wYvszjPA3avdy0F5h4Qmk9sKelfld5ymk1cKZcMtoP3BgRi8oN6xuB/WXZ2xGxujzVdNekbU21D0lSl5z3TCIiHqd5FnBVRIzSfErpS8ATEbEBeA34ZGm+D7gFGAHeAe4ByMxTEfEF4NnS7vOZOXEz/DM0n6BaAHy3vDjHPiRJXXLekMjMOyqL1kzRNoF7K9vZDmyfon4YuGaK+k+n2ockqXv8xLUkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmq6vhvXEu6+B0dO9PTPyWq2cszCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVNVRSETEH0fEsYh4ISIej4j3RMTyiDgUESMR8e2IuKS0vbTMj5Tly1q280Cp/ygibmqpD5faSERs7qSvkqQL13ZIRMRi4LPAYGZeA8wD1gFfBh7KzPcDp4ENZZUNwOlSf6i0IyJWlvU+CAwD34iIeRExD/g6cDOwErijtJUkdUmnl5vmAwsiYj7wXuAkcAOwuyzfCdxWpteWecryNRERpb4rM3+Rma8AI8B15TWSmccz811gV2krSeqStkMiM8eAPwN+TDMczgBHgLcyc7w0GwUWl+nFwImy7nhpf2VrfdI6tbokqUva/u6miFhE81/2y4G3gP9O83JR10XERmAjwMDAAI1Go63tnD17tu11ZyvH3B8GFsCmVePnbziH9ONxnokxd/IFf78HvJKZPwGIiO8AHwUWRsT8crawBBgr7ceApcBouTx1OfDTlvqE1nVq9X8iM7cCWwEGBwdzaGiorQE1Gg3aXXe2csz94WuP7eErR/vr+zx3DF/Wd8d5Jn63O7kn8WNgdUS8t9xbWAO8CDwD3F7arAf2lOm9ZZ6y/OnMzFJfV55+Wg6sAL4PPAusKE9LXULz5vbeDvorSbpAbf/TIjMPRcRu4AfAOPAczX/NPwnsiogvltq2sso24FsRMQKcovmmT2Yei4gnaAbMOHBvZv4SICLuA/bTfHJqe2Yea7e/kqQL19H5Z2ZuAbZMKh+n+WTS5LY/Bz5R2c6DwINT1PcB+zrpoySpfX7iWpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklTVX38ZXVLfODp2hrs3P9n1/b76pY91fZ8zyTMJSVKVISFJqjIkJElVhoQkqcqQkCRVGRKSpKqOQiIiFkbE7oj4YUS8FBG/ExFXRMSBiHi5/FxU2kZEPBwRIxHxfERc27Kd9aX9yxGxvqX+kYg4WtZ5OCKik/5Kki5Mp2cSXwX+JjN/G/gQ8BKwGXgqM1cAT5V5gJuBFeW1EXgEICKuALYA1wPXAVsmgqW0+XTLesMd9leSdAHaDomIuBz4XWAbQGa+m5lvAWuBnaXZTuC2Mr0WeDSbDgILI+Jq4CbgQGaeyszTwAFguCx7X2YezMwEHm3ZliSpCzr5xPVy4CfAX0bEh4AjwP3AQGaeLG1eBwbK9GLgRMv6o6V2rvroFPVfEREbaZ6dMDAwQKPRaGtAZ8+ebXvd2cox94eBBbBp1Xivu9FVvRpzL3+3ZuJ3u5OQmA9cC/xhZh6KiK/yj5eWAMjMjIjspIO/jszcCmwFGBwczKGhoba202g0aHfd2cox94evPbaHrxztr2/h2bRqvCdjfvXOoa7vc8JM/G53ck9iFBjNzENlfjfN0HijXCqi/HyzLB8Dlrasv6TUzlVfMkVdktQlbYdEZr4OnIiID5TSGuBFYC8w8YTSemBPmd4L3FWecloNnCmXpfYDN0bEonLD+kZgf1n2dkSsLk813dWyLUlSF3R6LvaHwGMRcQlwHLiHZvA8EREbgNeAT5a2+4BbgBHgndKWzDwVEV8Ani3tPp+Zp8r0Z4AdwALgu+UlSeqSjkIiM/8OGJxi0Zop2iZwb2U724HtU9QPA9d00kfpYrGsB19bPWHTqp7tWrOcn7iWJFUZEpKkKkNCklRlSEiSqgwJSVKVISFJqjIkJElVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVXzO91ARMwDDgNjmXlrRCwHdgFXAkeAT2XmuxFxKfAo8BHgp8DvZ+arZRsPABuAXwKfzcz9pT4MfBWYB3wzM7/UaX+lo2NnuHvzk73uhjQrTMeZxP3ASy3zXwYeysz3A6dpvvlTfp4u9YdKOyJiJbAO+CAwDHwjIuaV8Pk6cDOwErijtJUkdUlHIRERS4CPAd8s8wHcAOwuTXYCt5XptWWesnxNab8W2JWZv8jMV4AR4LryGsnM45n5Ls2zk7Wd9FeSdGE6vdz0F8DngH9e5q8E3srM8TI/Ciwu04uBEwCZOR4RZ0r7xcDBlm22rnNiUv36qToRERuBjQADAwM0Go22BnP27Nm2152t+nHMAwtg06rx8zecQxxz9/Ty/6eZ+P+57ZCIiFuBNzPzSEQMTV+XLlxmbgW2AgwODubQUHvdaTQatLvubNWPY/7aY3v4ytGOb8fNKptWjTvmLnn1zqGu73PCTPz/3Ml/wY8CH4+IW4D3AO+jeZN5YUTML2cTS4Cx0n4MWAqMRsR84HKaN7An6hNa16nVJUld0PY9icx8IDOXZOYymjeen87MO4FngNtLs/XAnjK9t8xTlj+dmVnq6yLi0vJk1Arg+8CzwIqIWB4Rl5R97G23v5KkCzcT52J/AuyKiC8CzwHbSn0b8K2IGAFO0XzTJzOPRcQTwIvAOHBvZv4SICLuA/bTfAR2e2Yem4H+SpIqpiUkMrMBNMr0cZpPJk1u83PgE5X1HwQenKK+D9g3HX2UJF04P3EtSaoyJCRJVYaEJKnKkJAkVfXXp2t00VjWwy/Y27SqZ7uWZh3PJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklRlSEiSqgwJSVKVf0+izx0dO8PdPfzbDpIubp5JSJKqDAlJUlXbIRERSyPimYh4MSKORcT9pX5FRByIiJfLz0WlHhHxcESMRMTzEXFty7bWl/YvR8T6lvpHIuJoWefhiIhOBitJujCdnEmMA5sycyWwGrg3IlYCm4GnMnMF8FSZB7gZWFFeG4FHoBkqwBbgeuA6YMtEsJQ2n25Zb7iD/kqSLlDbIZGZJzPzB2X6H4CXgMXAWmBnabYTuK1MrwUezaaDwMKIuBq4CTiQmacy8zRwABguy96XmQczM4FHW7YlSeqCaXm6KSKWAR8GDgEDmXmyLHodGCjTi4ETLauNltq56qNT1Kfa/0aaZycMDAzQaDTaGsfZs2fbXne2GlgAm1aN97obXeWY+0OvxtzL95CZeA/rOCQi4jeBvwL+KDPfbr1tkJkZEdnpPs4nM7cCWwEGBwdzaGiore00Gg3aXXe2+tpje/jK0f56EnrTqnHH3Ad6NeZX7xzq+j4nzMR7WEdPN0XEb9AMiMcy8zul/Ea5VET5+WapjwFLW1ZfUmrnqi+Zoi5J6pJOnm4KYBvwUmb+ecuivcDEE0rrgT0t9bvKU06rgTPlstR+4MaIWFRuWN8I7C/L3o6I1WVfd7VsS5LUBZ2ci30U+BRwNCL+rtT+FPgS8EREbABeAz5Zlu0DbgFGgHeAewAy81REfAF4trT7fGaeKtOfAXYAC4DvlpckqUvaDonM/N9A7XMLa6Zon8C9lW1tB7ZPUT8MXNNuHyVJnfET15KkKkNCklRlSEiSqgwJSVKVISFJqjIkJElVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqr66wvmL2LLNj/Zk/1uWtWT3UqaJTyTkCRVeSYhSdOoV1cFAHYMXzbt2/RMQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqP0zX4ujYGe7u4QdhJOlic9GfSUTEcET8KCJGImJzr/sjSf3kog6JiJgHfB24GVgJ3BERK3vbK0nqHxd1SADXASOZeTwz3wV2AWt73CdJ6huRmb3uQ1VE3A4MZ+YflPlPAddn5n2T2m0ENpbZDwA/anOXVwF/3+a6s5Vj7g+OuT90MuZ/lZm/Nbk4J25cZ+ZWYGun24mIw5k5OA1dmjUcc39wzP1hJsZ8sV9uGgOWtswvKTVJUhdc7CHxLLAiIpZHxCXAOmBvj/skSX3jor7clJnjEXEfsB+YB2zPzGMzuMuOL1nNQo65Pzjm/jDtY76ob1xLknrrYr/cJEnqIUNCklRlSBRz/es/ImJpRDwTES9GxLGIuL/Ur4iIAxHxcvm5qNd9nW4RMS8inouIvy7zyyPiUDnW3y4PRcwZEbEwInZHxA8j4qWI+J25fpwj4o/L7/ULEfF4RLxnrh3niNgeEW9GxAsttSmPazQ9XMb+fERc2+5+DQn65us/xoFNmbkSWA3cW8a4GXgqM1cAT5X5ueZ+4KWW+S8DD2Xm+4HTwIae9GrmfBX4m8z8beBDNMc+Z49zRCwGPgsMZuY1NB9yWcfcO847gOFJtdpxvRlYUV4bgUfa3akh0TTnv/4jM09m5g/K9D/QfONYTHOcO0uzncBtvenhzIiIJcDHgG+W+QBuAHaXJnNqzBFxOfC7wDaAzHw3M99ijh9nmk9qLoiI+cB7gZPMseOcmd8DTk0q147rWuDRbDoILIyIq9vZryHRtBg40TI/WmpzUkQsAz4MHAIGMvNkWfQ6MNCjbs2UvwA+B/y/Mn8l8FZmjpf5uXaslwM/Af6yXGL7ZkRcxhw+zpk5BvwZ8GOa4XAGOMLcPs4Tasd12t7TDIk+ExG/CfwV8EeZ+Xbrsmw+Dz1nnomOiFuBNzPzSK/70kXzgWuBRzLzw8DPmHRpaQ4e50U0/+W8HPgXwGX86mWZOW+mjqsh0dQXX/8REb9BMyAey8zvlPIbE6eh5eebverfDPgo8PGIeJXmJcQbaF6vX1guS8DcO9ajwGhmHirzu2mGxlw+zr8HvJKZP8nM/wt8h+axn8vHeULtuE7be5oh0TTnv/6jXIvfBryUmX/esmgvsL5Mrwf2dLtvMyUzH8jMJZm5jOYxfToz7wSeAW4vzebamF8HTkTEB0ppDfAic/g407zMtDoi3lt+zyfGPGePc4vacd0L3FWecloNnGm5LHVB/MR1ERG30Lx+PfH1Hw/2uEvTKiL+HfC/gKP84/X5P6V5X+IJ4F8CrwGfzMzJN8dmvYgYAv5TZt4aEf+a5pnFFcBzwL/PzF/0sn/TKSL+Lc0b9ZcAx4F7aP6DcM4e54j4L8Dv03yK7zngD2heg58zxzkiHgeGaH4d+BvAFuB/MsVxLWH532hednsHuCczD7e1X0NCklTj5SZJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklT1/wGCf1m1Nln3kgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "clusters_df.lexical_overlap.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    458569.000000\n",
       "mean          2.551876\n",
       "std         119.640853\n",
       "min           2.000000\n",
       "25%           2.000000\n",
       "50%           2.000000\n",
       "75%           2.000000\n",
       "max       80757.000000\n",
       "Name: cluster_size, dtype: float64"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_df.cluster_size.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>size</th>\n",
       "      <th>time_delta</th>\n",
       "      <th>lexical_overlap</th>\n",
       "      <th>cluster_size</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>458568.000000</td>\n",
       "      <td>458568</td>\n",
       "      <td>458568.000000</td>\n",
       "      <td>458568.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.375774</td>\n",
       "      <td>332 days 14:25:59.114460</td>\n",
       "      <td>62.386284</td>\n",
       "      <td>2.375774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.640274</td>\n",
       "      <td>1945 days 12:16:05.903994</td>\n",
       "      <td>18.888465</td>\n",
       "      <td>9.640274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0 days 00:00:00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0 days 00:00:00</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0 days 00:00:00</td>\n",
       "      <td>67.500000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>3 days 00:00:00</td>\n",
       "      <td>76.666667</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3256.000000</td>\n",
       "      <td>61971 days 00:00:00</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>3256.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                size                 time_delta lexical_overlap   cluster_size\n",
       "               count                                                          \n",
       "count  458568.000000                     458568   458568.000000  458568.000000\n",
       "mean        2.375774   332 days 14:25:59.114460       62.386284       2.375774\n",
       "std         9.640274  1945 days 12:16:05.903994       18.888465       9.640274\n",
       "min         2.000000            0 days 00:00:00        0.000000       2.000000\n",
       "25%         2.000000            0 days 00:00:00       50.000000       2.000000\n",
       "50%         2.000000            0 days 00:00:00       67.500000       2.000000\n",
       "75%         2.000000            3 days 00:00:00       76.666667       2.000000\n",
       "max      3256.000000        61971 days 00:00:00      100.000000    3256.000000"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_df[clusters_df.cluster_size < 50000].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date             min      1\n",
       "                 max      1\n",
       "size             count    1\n",
       "time_delta                1\n",
       "members                   1\n",
       "lexical_overlap           1\n",
       "cluster_size              1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_df[clusters_df.cluster_size < 50000].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f7a37b08a58>"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAATUElEQVR4nO3de5CddX3H8ffXhEsKSgI4O2nCNDhmdIK0CDtcRuusoBCQMfyhThimREvNTMGOVmYk1GmpF2bAlqJQi2YkNTiUkKI2DMWmKWZH2w63yCUETFlClGSQKOHS4DX22z+eX/RkOb/dPZvdc47yfs2c2ef5Ppffd/ecnM85z/Ock8hMJElq51W9bkCS1L8MCUlSlSEhSaoyJCRJVYaEJKlqZq8bmGpHH310LliwYFLbvvTSSxx22GFT29AU6Me++rEnsK9O9GNPYF+dmqq+Nm3a9KPMfO3LFmTmb9XtpJNOysnauHHjpLedTv3YVz/2lGlfnejHnjLtq1NT1Rdwf7Z5TvVwkySpypCQJFUZEpKkKkNCklRlSEiSqgwJSVKVISFJqjIkJElVhoQkqeq37ms5DsTmnS/w/hX/2vVxt1/1rq6PKUkT4TsJSVKVISFJqjIkJElVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkqgmHRETMiIgHIuKOMn9sRNwTESMRcWtEHFzqh5T5kbJ8Qcs+Li/1rRFxVkt9camNRMSKlnrbMSRJ3dHJO4kPA4+1zF8NXJuZrweeAy4q9YuA50r92rIeEbEIWAocBywG/qEEzwzg88DZwCLg/LLuWGNIkrpgQiEREfOBdwFfKvMBnA7cVlZZDZxXppeUecryM8r6S4A1mfmzzHwSGAFOLreRzNyWmT8H1gBLxhlDktQFMye43meBjwGvLvNHAc9n5t4yvwOYV6bnAU8BZObeiHihrD8PuLtln63bPDWqfso4Y+wnIpYDywEGBgYYHh6e4K+1v4FZcOnxe8dfcYqN1++ePXsm/TtNl37sCeyrE/3YE9hXp6a7r3FDIiLOBXZl5qaIGJq2Tg5AZq4EVgIMDg7m0NDQpPZz/c3ruGbzRHNz6my/YGjM5cPDw0z2d5ou/dgT2Fcn+rEnsK9OTXdfE3lGfAvw7og4BzgUeA3wOWB2RMwsr/TnAzvL+juBY4AdETETOAJ4tqW+T+s27erPjjGGJKkLxj0nkZmXZ+b8zFxAc+L5m5l5AbAReE9ZbRmwrkzfXuYpy7+ZmVnqS8vVT8cCC4F7gfuAheVKpoPLGLeXbWpjSJK64EA+J3EZ8NGIGKE5f3Bjqd8IHFXqHwVWAGTmFmAt8Cjwb8AlmfnL8i7hQ8B6mqun1pZ1xxpDktQFHR2Az8xhYLhMb6O5Mmn0Oj8F3lvZ/krgyjb1O4E729TbjiFJ6g4/cS1JqjIkJElVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklRlSEiSqgwJSVKVISFJqjIkJElVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVY0bEhFxaETcGxEPRcSWiPhEqR8bEfdExEhE3BoRB5f6IWV+pCxf0LKvy0t9a0Sc1VJfXGojEbGipd52DElSd0zkncTPgNMz8w+AE4DFEXEqcDVwbWa+HngOuKisfxHwXKlfW9YjIhYBS4HjgMXAP0TEjIiYAXweOBtYBJxf1mWMMSRJXTBuSGRjT5k9qNwSOB24rdRXA+eV6SVlnrL8jIiIUl+TmT/LzCeBEeDkchvJzG2Z+XNgDbCkbFMbQ5LUBTMnslJ5tb8JeD3Nq/4ngOczc29ZZQcwr0zPA54CyMy9EfECcFSp392y29ZtnhpVP6VsUxtjdH/LgeUAAwMDDA8PT+TXepmBWXDp8XvHX3GKjdfvnj17Jv07TZd+7AnsqxP92BPYV6emu68JhURm/hI4ISJmA18H3jhtHU1CZq4EVgIMDg7m0NDQpPZz/c3ruGbzhP4kU2r7BUNjLh8eHmayv9N06ceewL460Y89gX11arr76ujqpsx8HtgInAbMjoh9z6jzgZ1leidwDEBZfgTwbGt91Da1+rNjjCFJ6oKJXN302vIOgoiYBbwTeIwmLN5TVlsGrCvTt5d5yvJvZmaW+tJy9dOxwELgXuA+YGG5kulgmpPbt5dtamNIkrpgIsdW5gKry3mJVwFrM/OOiHgUWBMRnwYeAG4s698IfCUiRoDdNE/6ZOaWiFgLPArsBS4ph7GIiA8B64EZwKrM3FL2dVllDElSF4wbEpn5MPDmNvVtNFcmja7/FHhvZV9XAle2qd8J3DnRMSRJ3eEnriVJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklRlSEiSqgwJSVKVISFJqjIkJElVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkqnFDIiKOiYiNEfFoRGyJiA+X+pERsSEiHi8/55R6RMR1ETESEQ9HxIkt+1pW1n88Ipa11E+KiM1lm+siIsYaQ5LUHRN5J7EXuDQzFwGnApdExCJgBXBXZi4E7irzAGcDC8ttOXADNE/4wBXAKcDJwBUtT/o3AB9s2W5xqdfGkCR1wbghkZlPZ+Z3yvT/Ao8B84AlwOqy2mrgvDK9BLgpG3cDsyNiLnAWsCEzd2fmc8AGYHFZ9prMvDszE7hp1L7ajSFJ6oJonpcnuHLEAuBbwJuA72fm7FIP4LnMnB0RdwBXZeZ/lmV3AZcBQ8ChmfnpUv9L4CfAcFn/HaX+h8BlmXluRDzfbow2fS2nedfCwMDASWvWrOnwz9DYtfsFnvnJpDY9IMfPO2LM5Xv27OHwww/vUjcT0489gX11oh97Avvq1FT19fa3v31TZg6Ors+c6A4i4nDgq8BHMvPFctoAgMzMiJh42kzCWGNk5kpgJcDg4GAODQ1Naozrb17HNZsn/CeZMtsvGBpz+fDwMJP9naZLP/YE9tWJfuwJ7KtT093XhK5uioiDaALi5sz8Wik/Uw4VUX7uKvWdwDEtm88vtbHq89vUxxpDktQFE7m6KYAbgccy8+9aFt0O7LtCaRmwrqV+YbnK6VTghcx8GlgPnBkRc8oJ6zOB9WXZixFxahnrwlH7ajeGJKkLJnJs5S3AHwGbI+LBUvsL4CpgbURcBHwPeF9ZdidwDjAC/Bj4AEBm7o6ITwH3lfU+mZm7y/TFwJeBWcA3yo0xxpAkdcG4IVFOQEdl8Rlt1k/gksq+VgGr2tTvpzkZPrr+bLsxJEnd4SeuJUlVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklRlSEiSqgwJSVKVISFJqjIkJElVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqSqcUMiIlZFxK6IeKSldmREbIiIx8vPOaUeEXFdRIxExMMRcWLLNsvK+o9HxLKW+kkRsblsc11ExFhjSJK6ZyLvJL4MLB5VWwHclZkLgbvKPMDZwMJyWw7cAM0TPnAFcApwMnBFy5P+DcAHW7ZbPM4YkqQuGTckMvNbwO5R5SXA6jK9GjivpX5TNu4GZkfEXOAsYENm7s7M54ANwOKy7DWZeXdmJnDTqH21G0OS1CWTPScxkJlPl+kfAANleh7wVMt6O0ptrPqONvWxxpAkdcnMA91BZmZE5FQ0M9kxImI5zeEtBgYGGB4entQ4A7Pg0uP3TmrbAzFev3v27Jn07zRd+rEnsK9O9GNPYF+dmu6+JhsSz0TE3Mx8uhwy2lXqO4FjWtabX2o7gaFR9eFSn99m/bHGeJnMXAmsBBgcHMyhoaHaqmO6/uZ1XLP5gHOzY9svGBpz+fDwMJP9naZLP/YE9tWJfuwJ7KtT093XZA833Q7su0JpGbCupX5hucrpVOCFcshoPXBmRMwpJ6zPBNaXZS9GxKnlqqYLR+2r3RiSpC4Z92VzRNxC8y7g6IjYQXOV0lXA2oi4CPge8L6y+p3AOcAI8GPgAwCZuTsiPgXcV9b7ZGbuOxl+Mc0VVLOAb5QbY4whSeqScUMiM8+vLDqjzboJXFLZzypgVZv6/cCb2tSfbTeGJKl7/MS1JKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklRlSEiSqgwJSVKVISFJqjIkJElVhoQkqcqQkCRVGRKSpCpDQpJUZUhIkqoMCUlSlSEhSaoyJCRJVYaEJKnKkJAkVRkSkqQqQ0KSVGVISJKqDAlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklRlSEiSqvo+JCJicURsjYiRiFjR634k6ZWkr0MiImYAnwfOBhYB50fEot52JUmvHH0dEsDJwEhmbsvMnwNrgCU97kmSXjFm9rqBccwDnmqZ3wGcMnqliFgOLC+zeyJi6yTHOxr40SS3nbS4etxVetLXOPqxJ7CvTvRjT2BfnZqqvn6vXbHfQ2JCMnMlsPJA9xMR92fm4BS0NKX6sa9+7AnsqxP92BPYV6emu69+P9y0EzimZX5+qUmSuqDfQ+I+YGFEHBsRBwNLgdt73JMkvWL09eGmzNwbER8C1gMzgFWZuWUahzzgQ1bTpB/76seewL460Y89gX11alr7isyczv1Lkn6D9fvhJklSDxkSkqQqQ6KY7q//iIhVEbErIh5pqR0ZERsi4vHyc06pR0RcV3p5OCJObNlmWVn/8YhY1lI/KSI2l22ui4iYQE/HRMTGiHg0IrZExIf7pK9DI+LeiHio9PWJUj82Iu4p+7q1XMxARBxS5kfK8gUt+7q81LdGxFkt9Unf3xExIyIeiIg7+qWviNhe/s4PRsT9pdbr+3F2RNwWEd+NiMci4rQ+6OkN5W+07/ZiRHyk132V7f68PN4fiYhbovl30PPHFpn5ir/RnBR/AngdcDDwELBoisd4G3Ai8EhL7TPAijK9Ari6TJ8DfAMI4FTgnlI/EthWfs4p03PKsnvLulG2PXsCPc0FTizTrwb+h+brT3rdVwCHl+mDgHvKPtYCS0v9C8CflumLgS+U6aXArWV6UbkvDwGOLffxjAO9v4GPAv8E3FHme94XsB04elSt1/fjauBPyvTBwOxe99Tm3/0PaD5E1uu/1TzgSWBWy2Pq/X3x2DqQJ77flhtwGrC+Zf5y4PJpGGcB+4fEVmBumZ4LbC3TXwTOH70ecD7wxZb6F0ttLvDdlvp+63XQ3zrgnf3UF/A7wHdoPmn/I2Dm6PuM5uq308r0zLJejL4f9613IPc3zWd17gJOB+4o4/RDX9t5eUj07H4EjqB50ot+6alNj2cC/9UPffHrb5c4sjxW7gDO6ofHloebGu2+/mNeF8YdyMyny/QPgIFx+hmrvqNNfcLK29U307xq73lf0RzSeRDYBWygeRX0fGbubbOvX41flr8AHDWJfifis8DHgP8r80f1SV8J/HtEbIrma2qgt/fjscAPgX+M5tDclyLisB73NNpS4JYy3dO+MnMn8LfA94GnaR4rm+iDx5Yh0SeyifeeXI8cEYcDXwU+kpkv9kNfmfnLzDyB5pX7ycAbu93DaBFxLrArMzf1upc23pqZJ9J8Y/IlEfG21oU9uB9n0hxevSEz3wy8RHMYp5c9/Uo5tv9u4J9HL+tFX+UcyBKacP1d4DBgcTd7qDEkGr36+o9nImIuQPm5a5x+xqrPb1MfV0QcRBMQN2fm1/qlr30y83lgI83b5dkRse8DoK37+tX4ZfkRwLOT6Hc8bwHeHRHbab6R+HTgc33Q175XomTmLuDrNMHay/txB7AjM+8p87fRhEa/PLbOBr6Tmc+U+V739Q7gycz8YWb+AvgazeOt54+tKT3m/pt6o3nVs40mxfed1DluGsZZwP7nJP6G/U+WfaZMv4v9T5bdW+pH0hznnVNuTwJHlmWjT5adM4F+ArgJ+Oyoeq/7ei0wu0zPAr4NnEvzqq/1JN7FZfoS9j+Jt7ZMH8f+J/G20ZzAO+D7Gxji1yeue9oXzavOV7dM/zfNq9Be34/fBt5Qpv+69NPTnlp6WwN8oI8e86cAW2jOwQXNSf8/6/VjK9MT16130jk0V/c8AXx8GvZ/C82xxl/QvMq6iOYY4l3A48B/tDzIguY/W3oC2AwMtuznj4GRcmt9kA8Cj5Rt/p5RJwwrPb2V5m31w8CD5XZOH/T1+8ADpa9HgL8q9deVf4Aj5R/PIaV+aJkfKctf17Kvj5ext9JylcmB3t/sHxI97auM/1C5bdm3XR/cjycA95f78V9onkx72lPZ7jCaV91HtNT6oa9PAN8t236F5om+5495v5ZDklTlOQlJUpUhIUmqMiQkSVWGhCSpypCQJFUZEpKkKkNCklT1//JmPv7uIieUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "clusters_df.cluster_size.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f7a37c4f198>"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAASAklEQVR4nO3df4wcd3nH8fdTBweUoyEQdEJOVDu9KK0VV5CsCFER2msLtgnml6LWVqQSasWC1lUrUQlbVJX5o4JWSv9ISBtdRRRauT7clDZx4iqEllP5I4XEbcA2lsFQo9iivoLB7UVWqeHpHzeG5XJn7+2Pm51v3i/p5J3vzsx+Zm7zZO7Z2ZnITCRJZfmZugNIkgbP4i5JBbK4S1KBLO6SVCCLuyQV6Iq6AwBce+21uXbt2p6WfeGFF7jqqqsGG2iFmH3lNTU3mL0uo5z90KFD38nM1y723EgU97Vr1/Lss8/2tOzMzAztdnuwgVaI2VdeU3OD2esyytkj4ltLPVdrWyYitkTE1Llz5+qMIUnFqbW4Z+aBzNxx9dVX1xlDkorjkbskFcgjd0kqkEfuklQgj9wlqUB+iUmSCmRbRpIKVOuXmDLzAHCg1Wrd0+s6Dp8+x927nhhgqu6d/PgdtbyuJF2ObRlJKpBtGUkqkGfLSFKBbMtIUoEs7pJUIIu7JBXID1QlqUB+oCpJBbItI0kFsrhLUoEs7pJUIIu7JBXIs2UkqUCeLSNJBbItI0kFsrhLUoEs7pJUIIu7JBXI4i5JBRp4cY+IdkR8ISIejIj2oNcvSbq8rop7RDwUEbMRcWTB+KaIOB4RJyJiVzWcwBzwcuDUYONKkrrR7ZH7w8CmzoGIWAU8AGwG1gPbImI98IXM3Ax8GPjo4KJKkroVmdndjBFrgccz8+Zq+nZgT2ZurKZ3A2Tmx6rp1cDfZOadS6xvB7ADYHx8/Nbp6emeNmD27DnOnO9p0b5tWNPfl6/m5uYYGxsbUJqV1dTsTc0NZq/LKGefnJw8lJmtxZ67oo/1rgGe75g+BdwWEe8FNgKvAj6x1MKZOQVMAbRarWy32z2FuH/vo9x7uJ/N6N3Ju9p9LT8zM0Ov2123pmZvam4we12amn3gVTEzPwN8ppt5I2ILsGViYmLQMSTpJa2fs2VOA9d3TF9XjXXNa8tI0nD0U9yfAW6MiHVVf30r8NhyVuBVISVpOLo9FXIf8DRwU0SciojtmXkB2Ak8CRwD9mfm0eW8uEfukjQcXfXcM3PbEuMHgYO9vrg9d0kaDq/nLkkF8k5MklQgj9wlqUBeFVKSCmRbRpIKZFtGkgpkW0aSCmRbRpIKZFtGkgpkW0aSCmRxl6QCWdwlqUB+oCpJBfIDVUkqkG0ZSSqQxV2SCmRxl6QCWdwlqUCeLSNJBfJsGUkqkG0ZSSqQxV2SCmRxl6QCWdwlqUAWd0kq0FCKe0RcFRHPRsQ7hrF+SdKldVXcI+KhiJiNiCMLxjdFxPGIOBERuzqe+jCwf5BBJUnd6/bI/WFgU+dARKwCHgA2A+uBbRGxPiLeCnwVmB1gTknSMkRmdjdjxFrg8cy8uZq+HdiTmRur6d3VrGPAVcwX/PPAezLzR4usbwewA2B8fPzW6enpnjZg9uw5zpzvadG+bVjT35ev5ubmGBsbG1CaldXU7E3NDWavyyhnn5ycPJSZrcWeu6KP9a4Bnu+YPgXclpk7ASLibuA7ixV2gMycAqYAWq1WttvtnkLcv/dR7j3cz2b07uRd7b6Wn5mZodftrltTszc1N5i9Lk3NPrSqmJkPX26eiNgCbJmYmBhWDEl6SernbJnTwPUd09dVY13z2jKSNBz9FPdngBsjYl1ErAa2Ao8tZwVeFVKShqPbUyH3AU8DN0XEqYjYnpkXgJ3Ak8AxYH9mHl3Oi3vkLknD0VXPPTO3LTF+EDjY64vbc5ek4fB67pJUIO/EJEkF8shdkgrkVSElqUC2ZSSpQLZlJKlAtmUkqUC2ZSSpQLZlJKlAtmUkqUAWd0kqkMVdkgrkB6qSVCA/UJWkAtmWkaQCWdwlqUAWd0kqkMVdkgrk2TKSVCDPlpGkAtmWkaQCWdwlqUAWd0kqkMVdkgpkcZekAg28uEfEL0bEgxHxSER8cNDrlyRdXlfFPSIeiojZiDiyYHxTRByPiBMRsQsgM49l5geAXwd+efCRJUmX0+2R+8PAps6BiFgFPABsBtYD2yJiffXcO4EngIMDSypJ6lpkZnczRqwFHs/Mm6vp24E9mbmxmt4NkJkf61jmicy8Y4n17QB2AIyPj986PT3d0wbMnj3HmfM9Ldq3DWv6+/LV3NwcY2NjA0qzspqavam5wex1GeXsk5OThzKztdhzV/Sx3jXA8x3Tp4DbIqINvBe4kkscuWfmFDAF0Gq1st1u9xTi/r2Pcu/hfjajdyfvave1/MzMDL1ud92amr2pucHsdWlq9oFXxcycAWa6mTcitgBbJiYmBh1Dkl7S+jlb5jRwfcf0ddVY17y2jCQNRz/F/RngxohYFxGrga3AY8tZgVeFlKTh6PZUyH3A08BNEXEqIrZn5gVgJ/AkcAzYn5lHl/PiHrlL0nB01XPPzG1LjB+kj9Md7blL0nB4PXdJKpB3YpKkAnnkLkkF8qqQklQg2zKSVCDbMpJUINsyklQg2zKSVCDbMpJUINsyklQgi7skFcieuyQVyJ67JBXItowkFcjiLkkFsrhLUoEs7pJUIIu7JBXIUyElqUCeCilJBbItI0kFsrhLUoEs7pJUIIu7JBXI4i5JBbpiGCuNiHcDdwA/C3wyMz87jNeRJC2u6yP3iHgoImYj4siC8U0RcTwiTkTELoDM/IfMvAf4APAbg40sSbqc5bRlHgY2dQ5ExCrgAWAzsB7YFhHrO2b5w+p5SdIKiszsfuaItcDjmXlzNX07sCczN1bTu6tZP179PJWZn1tiXTuAHQDj4+O3Tk9P97QBs2fPceZ8T4v2bcOa/r58NTc3x9jY2IDSrKymZm9qbjB7XUY5++Tk5KHMbC32XL899zXA8x3Tp4DbgN8Ffg24OiImMvPBhQtm5hQwBdBqtbLdbvcU4P69j3Lv4aF8dHBZJ+9q97X8zMwMvW533Zqavam5wex1aWr2oVTFzLwPuO9y80XEFmDLxMTEMGJI0ktWv6dCngau75i+rhqTJNWo3+L+DHBjRKyLiNXAVuCxbhf2wmGSNBzLORVyH/A0cFNEnIqI7Zl5AdgJPAkcA/Zn5tFlrNNL/krSEHTdc8/MbUuMHwQO9vLimXkAONBqte7pZXlJ0uK8/IAkFcg7MUlSgbwTkyQVyLaMJBXItowkFci2jCQVyLaMJBXItowkFci2jCQVyLaMJBXI4i5JBbLnLkkFsucuSQWyLSNJBbK4S1KBLO6SVCCLuyQVyLNlJKlAni0jSQWyLSNJBbK4S1KBLO6SVCCLuyQVyOIuSQWyuEtSga4Y9Aoj4gbgI8DVmXnnoNc/StbueqKv5T+04QJ397COkx+/o6/XlVS+ro7cI+KhiJiNiCMLxjdFxPGIOBERuwAy85uZuX0YYSVJ3em2LfMwsKlzICJWAQ8Am4H1wLaIWD/QdJKknkRmdjdjxFrg8cy8uZq+HdiTmRur6d0AmfmxavqRS7VlImIHsANgfHz81unp6Z42YPbsOc6c72nR2o2/gp6yb1hT/zd65+bmGBsbqzvGsjU1N5i9LqOcfXJy8lBmthZ7rp+e+xrg+Y7pU8BtEfEa4I+BN0TE7ovFfqHMnAKmAFqtVrbb7Z5C3L/3Ue49PPCPDlbEhzZc6Cn7ybvagw+zTDMzM/T6O6tTU3OD2evS1OwDr4qZ+V3gA93MGxFbgC0TExODjiFJL2n9nAp5Gri+Y/q6akySVLN+ivszwI0RsS4iVgNbgceWswKvCilJw9HtqZD7gKeBmyLiVERsz8wLwE7gSeAYsD8zjy7nxb2euyQNR1c998zctsT4QeBgry+emQeAA61W655e1yFJejEvPyBJBfI2e5JUIG+zJ0kFsi0jSQWyLSNJBbItI0kFsi0jSQWq9YpbXlumN/3eJKQf3ihEagbbMpJUINsyklQgi7skFchTISWpQPbcJalAtmUkqUAWd0kqkMVdkgpkcZekAvkNVUkjo65vX5f4zWvPlpGkAtmWkaQCWdwlqUAWd0kqkMVdkgpkcZekAlncJalAAz/PPSKuAv4c+AEwk5l7B/0akqRL6+rIPSIeiojZiDiyYHxTRByPiBMRsasafi/wSGbeA7xzwHklSV3oti3zMLCpcyAiVgEPAJuB9cC2iFgPXAc8X832w8HElCQtR2RmdzNGrAUez8ybq+nbgT2ZubGa3l3Negr4XmY+HhHTmbl1ifXtAHYAjI+P3zo9Pd3TBsyePceZ8z0tWrvxV2D2Fdak3BvW/PQ3t+fm5hgbG6spTX+6zX749OjduGfY75mFv+flmJycPJSZrcWe66fnvoafHKHDfFG/DbgP+ERE3AEcWGrhzJwCpgBarVa22+2eQty/91HuPVzrJXJ69qENF8y+wpqU++Rd7Z+anpmZodf/TurWbfa7a7q2zKUM+z2z8Pc8KANPnJkvAO/vZl4vHCZJw9HPqZCnges7pq+rxiRJNeunuD8D3BgR6yJiNbAVeGw5K/CqkJI0HN2eCrkPeBq4KSJORcT2zLwA7ASeBI4B+zPz6HJePCK2RMTUuXOj9yGKJDVZVz33zNy2xPhB4GCvL56ZB4ADrVbrnl7XIUl6MS8/IEkFqrW425aRpOHwNnuSVKCuv6E61BAR/wV8q8fFrwW+M8A4K8nsK6+pucHsdRnl7D+Xma9d7ImRKO79iIhnl/r67agz+8pram4we12amt0PVCWpQBZ3SSpQCcV9qu4AfTD7ymtqbjB7XRqZvfE9d0nSi5Vw5C5JWsDiLkkFanRxX+IeriMjIk5GxOGIeC4inq3GXh0RT0XE16t/r6nGIyLuq7blKxFxywpnfdF9cnvJGhHvq+b/ekS8r8bseyLidLXvn4uIt3c8t7vKfjwiNnaMr+j7KSKuj4jPR8RXI+JoRPxeNT7y+/0S2Zuw318eEV+KiC9X2T9aja+LiC9WOT5dXe2WiLiymj5RPb/2cts0EjKzkT/AKuAbwA3AauDLwPq6cy3IeBK4dsHYnwK7qse7gD+pHr8d+EcggDcBX1zhrG8BbgGO9JoVeDXwzerfa6rH19SUfQ/wB4vMu756r1wJrKveQ6vqeD8BrwNuqR6/EvhalW/k9/slsjdhvwcwVj1+GfDFan/uB7ZW4w8CH6we/zbwYPV4K/DpS23TsN/v3f40+cj9jcCJzPxmZv4AmAbeVXOmbrwL+FT1+FPAuzvG/yrn/Svwqoh43UqFysx/Ac4uGF5u1o3AU5l5NjO/BzzFghurr2D2pbwLmM7M/83M/wBOMP9eWvH3U2Z+OzP/rXr8P8xfOnsNDdjvl8i+lFHa75mZc9Xky6qfBH4FeKQaX7jfL/4+HgF+NSLiEts0Eppc3Be7h+ul3lx1SOCzEXEo5m8IDjCemd+uHv8nMF49HsXtWW7WUduGnVX74qGLrQ1GNHv1p/4bmD+KbNR+X5AdGrDfI2JVRDwHzDL/P8NvAN/P+ftULMzx44zV8+eA19SVvVtNLu5N8ObMvAXYDPxORLyl88mc/9uuEeeiNilr5S+AnwdeD3wbuLfeOEuLiDHg74Dfz8z/7nxu1Pf7Itkbsd8z84eZ+Xrmbw/6RuAXao40cE0u7iN/D9fMPF39Owv8PfNvojMX2y3Vv7PV7KO4PcvNOjLbkJlnqv+AfwT8JT/5c3mkskfEy5gvjnsz8zPVcCP2+2LZm7LfL8rM7wOfB25nvs118QZGnTl+nLF6/mrgu4zQ+30xTS7ufd/DdZgi4qqIeOXFx8DbgCPMZ7x4NsP7gEerx48Bv1mdEfEm4FzHn+Z1WW7WJ4G3RcQ11Z/jb6vGVtyCzyvew/y+h/nsW6szINYBNwJfoob3U9W3/SRwLDP/rOOpkd/vS2VvyH5/bUS8qnr8CuCtzH9m8Hngzmq2hfv94u/jTuCfq7+oltqm0VD3J7r9/DB/9sDXmO+XfaTuPAuy3cD8J+lfBo5ezMd8r+6fgK8DnwNenT/5BP+BalsOA60VzruP+T+j/4/53uH2XrICv8X8B0sngPfXmP2vq2xfYf4/wtd1zP+RKvtxYHNd7yfgzcy3XL4CPFf9vL0J+/0S2Zuw338J+Pcq4xHgj6rxG5gvzieAvwWurMZfXk2fqJ6/4XLbNAo/Xn5AkgrU5LaMJGkJFndJKpDFXZIKZHGXpAJZ3CWpQBZ3SSqQxV2SCvT/IfvyzFG6xkIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "clusters_df[clusters_df.cluster_size < 50000].cluster_size.hist(log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f7aacd9f550>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAS80lEQVR4nO3da4wdd3nH8e9TGwfw0lwIWqV2hB1t5NYkFeCjBguEdiuobYLTFqHWVl4Q6sbiElTaSsUWVRVeVFDaSDQhbbDaNLQKWdw0JbHjKlyUlVAbQWJudggGA25jC7yEFKONIpXA0xdnEh9vduM5N5/x39+PZOXMf26/4zP7ePKc2ZnITCRJZfmlUQeQJA2exV2SCmRxl6QCWdwlqUAWd0kq0NJRBwC4+OKLc9WqVT2t+9RTT7F8+fLBBhoAc9XXxExgrm40MROUn2v//v1PZOYrFpyZmSP/s27duuzVgw8+2PO6w2Su+pqYKdNc3WhipszycwGP5CJ1daRtmYjYHBG7Tpw4McoYklSckRb3zNyTmdvPP//8UcaQpOJ45i5JBfLMXZIK5Jm7JBXIM3dJKpC/xCRJBbItI0kFGulvqGbmHmBPq9W6vtdtHDh2gut23D/AVPUd+cjVI9mvJJ2ObRlJKpBtGUkqkFfLSFKBbMtIUoEs7pJUIIu7JBXIL1QlqUB+oSpJBbItI0kFsrhLUoEs7pJUIIu7JBXIq2UkqUBeLSNJBbItI0kFsrhLUoEs7pJUIIu7JBXI4i5JBRp4cY+IyYj4YkTcFhGTg96+JOn0ahX3iLg9ImYj4uC88Y0RcSgiDkfEjmo4gTngxcDRwcaVJNVR98z9DmBj50BELAFuBTYBa4GtEbEW+GJmbgI+AHxocFElSXVFZtZbMGIVsDczr6im1wM3ZuaGanonQGZ+uJpeBnwqM9++yPa2A9sBxsfH101PT/f0BmafPMHxp3tatW9Xrlj8l6/m5uYYGxs7g2nqaWKuJmYCc3WjiZmg/FxTU1P7M7O10LylfWx3BfB4x/RR4KqIeBuwAbgA+PhiK2fmLmAXQKvVysnJyZ5C3HLnvdx0oJ+30bsj104uOm9mZoZe39MwNTFXEzOBubrRxExwbucaeFXMzHuAe+osGxGbgc0TExODjiFJ57R+rpY5BlzaMb2yGqvNe8tI0nD0U9wfBi6PiNVVf30LcF83G/CukJI0HHUvhbwLeAhYExFHI2JbZj4D3AA8ADwG7M7MR7vZuWfukjQctXrumbl1kfF9wL5ed27PXZKGw/u5S1KBfBKTJBXIM3dJKpB3hZSkAtmWkaQC2ZaRpALZlpGkAtmWkaQC2ZaRpALZlpGkAlncJalAFndJKpBfqEpSgfxCVZIKZFtGkgpkcZekAlncJalAFndJKpBXy0hSgbxaRpIKZFtGkgpkcZekAlncJalAFndJKpDFXZIKNJTiHhHLI+KRiHjrMLYvSXphtYp7RNweEbMRcXDe+MaIOBQRhyNiR8esDwC7BxlUklRf3TP3O4CNnQMRsQS4FdgErAW2RsTaiHgz8E1gdoA5JUldiMyst2DEKmBvZl5RTa8HbszMDdX0zmrRMWA57YL/NPC7mfmLBba3HdgOMD4+vm56erqnNzD75AmOP93Tqn27csXiv3w1NzfH2NjYGUxTTxNzNTETmKsbTcwE5eeampran5mtheYt7WO7K4DHO6aPAldl5g0AEXEd8MRChR0gM3cBuwBarVZOTk72FOKWO+/lpgP9vI3eHbl2ctF5MzMz9PqehqmJuZqYCczVjSZmgnM719CqYmbecbplImIzsHliYmJYMSTpnNTP1TLHgEs7pldWY7V5bxlJGo5+ivvDwOURsToilgFbgPu62YB3hZSk4ah7KeRdwEPAmog4GhHbMvMZ4AbgAeAxYHdmPtrNzj1zl6ThqNVzz8yti4zvA/b1unN77pI0HN7PXZIK5JOYJKlAnrlLUoG8K6QkFci2jCQVyLaMJBXItowkFci2jCQVyLaMJBXItowkFcjiLkkFsrhLUoH8QlWSCuQXqpJUINsyklQgi7skFcjiLkkFsrhLUoG8WkaSCuTVMpJUINsyklQgi7skFcjiLkkFsrhLUoEs7pJUoIEX94j4tYi4LSLujoh3D3r7kqTTq1XcI+L2iJiNiIPzxjdGxKGIOBwROwAy87HMfBfwe8DrBx9ZknQ6dc/c7wA2dg5ExBLgVmATsBbYGhFrq3nXAPcD+waWVJJUW2RmvQUjVgF7M/OKano9cGNmbqimdwJk5oc71rk/M69eZHvbge0A4+Pj66anp3t6A7NPnuD40z2t2rcrVyz+y1dzc3OMjY2dwTT1NDFXEzOBubrRxExQfq6pqan9mdlaaN7SPra7Ani8Y/oocFVETAJvA87jBc7cM3MXsAug1Wrl5ORkTyFuufNebjrQz9vo3ZFrJxedNzMzQ6/vaZiamKuJmcBc3WhiJji3cw28KmbmDDBTZ9mI2AxsnpiYGHQMSTqn9XO1zDHg0o7pldVYbd5bRpKGo5/i/jBweUSsjohlwBbgvm424F0hJWk46l4KeRfwELAmIo5GxLbMfAa4AXgAeAzYnZmPdrNzz9wlaThq9dwzc+si4/vo43JHe+6SNBzez12SCuSTmCSpQJ65S1KBvCukJBXItowkFci2jCQVyLaMJBXItowkFci2jCQVyLaMJBXI4i5JBbLnLkkFsucuSQWyLSNJBbK4S1KBLO6SVCCLuyQVyOIuSQXyUkhJKpCXQkpSgWzLSFKBLO6SVCCLuyQVyOIuSQWyuEtSgZYOY6MR8TvA1cAvA/+YmZ8dxn4kSQurfeYeEbdHxGxEHJw3vjEiDkXE4YjYAZCZn8nM64F3Ab8/2MiSpNPppi1zB7CxcyAilgC3ApuAtcDWiFjbscifV/MlSWdQZGb9hSNWAXsz84pqej1wY2ZuqKZ3Vot+pPrzucz8/CLb2g5sBxgfH183PT3d0xuYffIEx5/uadW+Xbli8V++mpubY2xs7AymqaeJuZqYCczVjSZmgvJzTU1N7c/M1kLz+u25rwAe75g+ClwFvA94E3B+RExk5m3zV8zMXcAugFarlZOTkz0FuOXOe7npwFC+OjitI9dOLjpvZmaGXt/TMDUxVxMzgbm60cRMcG7nGkpVzMybgZtPt1xEbAY2T0xMDCOGJJ2z+r0U8hhwacf0ympMkjRC/Rb3h4HLI2J1RCwDtgD31V3ZG4dJ0nB0cynkXcBDwJqIOBoR2zLzGeAG4AHgMWB3Zj7axTa95a8kDUHtnntmbl1kfB+wr5edZ+YeYE+r1bq+l/UlSQvz9gOSVCCfxCRJBfJJTJJUINsyklQg2zKSVCDbMpJUINsyklQg2zKSVCDbMpJUINsyklQgi7skFcieuyQVyJ67JBXItowkFcjiLkkFsrhLUoEs7pJUIK+WkaQCebWMJBXItowkFcjiLkkFsrhLUoEs7pJUIIu7JBXI4i5JBRp4cY+IyyLiHyPi7kFvW5JUT63iHhG3R8RsRBycN74xIg5FxOGI2AGQmd/LzG3DCCtJqqfumfsdwMbOgYhYAtwKbALWAlsjYu1A00mSehKZWW/BiFXA3sy8oppeD9yYmRuq6Z0AmfnhavruzHz7C2xvO7AdYHx8fN309HRPb2D2yRMcf7qnVft25YrFf7N2bm6OsbGxM5imnibmamImMFc3mpgJys81NTW1PzNbC81b2sd2VwCPd0wfBa6KiJcDfwm8JiJ2Plvs58vMXcAugFarlZOTkz2FuOXOe7npQD9vo3dHrp1cdN7MzAy9vqdhamKuJmYCc3WjiZng3M418KqYmT8G3lVn2YjYDGyemJgYdAxJOqf1c7XMMeDSjumV1ZgkacT6Ke4PA5dHxOqIWAZsAe7rZgPeFVKShqPupZB3AQ8BayLiaERsy8xngBuAB4DHgN2Z+Wg3O/d+7pI0HLV67pm5dZHxfcC+XneemXuAPa1W6/petyFJej5vPyBJBfIxe5JUIB+zJ0kFsi0jSQWyLSNJBbItI0kFsi0jSQWyLSNJBbItI0kFsi0jSQWyuEtSgey5S1KB7LlLUoFsy0hSgSzuklQgi7skFcjiLkkF8moZSSqQV8tIUoFsy0hSgSzuklQgi7skFcjiLkkFsrhLUoEs7pJUoKWD3mBELAf+Dvg/YCYz7xz0PiRJL6zWmXtE3B4RsxFxcN74xog4FBGHI2JHNfw24O7MvB64ZsB5JUk11G3L3AFs7ByIiCXArcAmYC2wNSLWAiuBx6vFfj6YmJKkbkRm1lswYhWwNzOvqKbXAzdm5oZqeme16FHgfzNzb0RMZ+aWRba3HdgOMD4+vm56erqnNzD75AmOP93Tqn27csXiv1k7NzfH2NjYGUxTTxNzNTETmKsbTcwE9XMdOHZmb4Ey/hKeq1svVEdOZ2pqan9mthaa10/PfQUnz9ChXdSvAm4GPh4RVwN7Fls5M3cBuwBarVZOTk72FOKWO+/lpgMD/+qgliPXTi46b2Zmhl7f0zA1MVcTM4G5utHETFA/13U77h9+mA5/euUzz9WtF6oj/Rh4VczMp4B31lk2IjYDmycmJgYdQ5LOaf1cCnkMuLRjemU1JkkasX6K+8PA5RGxOiKWAVuA+7rZgHeFlKThqHsp5F3AQ8CaiDgaEdsy8xngBuAB4DFgd2Y+2s3OvZ+7JA1HrZ57Zm5dZHwfsK/XnWfmHmBPq9W6vtdtSJKez9sPSFKBfMyeJBXIx+xJUoFq/4bqUENE/Aj47x5Xvxh4YoBxBsVc9TUxE5irG03MBOXnemVmvmKhGY0o7v2IiEcW+/XbUTJXfU3MBObqRhMzwbmdyy9UJalAFndJKlAJxX3XqAMswlz1NTETmKsbTcwE53Cus77nLkl6vhLO3CVJ81jcJalEmXnW/qH96L9DwGFgxxC2fzswCxzsGLsI+Bzwneq/F1bjQftBJYeBbwCv7VjnHdXy3wHe0TG+DjhQrXMzVZusRq5LgQeBbwKPAn806mzAi4EvA1+vMn2oGl8NfKnazqeBZdX4edX04Wr+qo5t7azGDwEbBvF5A0uAr9J+mlgjcgFHqr/jrwGPjPozrNa5ALgb+BbtGwKub0CmNdXf0bN/fgq8f9S5qvX+mPbxfhC4i/bPwciPrcw8e4s77R/W7wKXActoF5W1A97HG4HXcmpx/+izf8nADuCvqtdvAf6jOrBeB3wpT/6wfq/674XV62cPwi9Xy0a17qaauS559oAFXgZ8m/ZzbEeWrVpurHr9ourgfR2wG9hSjd8GvLt6/R7gtur1FuDT1eu11Wd5XvVD8t3qs+7r8wb+BPgUJ4v7yHPRLu4Xzxsb6fEFfBL4w+r1MtrFfuTH/Lyf+x8Crxx1LtpPo/s+8JKOY+q6JhxbmWd3cV8PPNAxvRPYOYT9rOLU4n4IuKR6fQlwqHr9CWDr/OWArcAnOsY/UY1dAnyrY/yU5brMeC/w5qZkA14KfIX2YxefAJbO/8xo3yp6ffV6abVczP8cn12un8+b9oNkvgD8JrC32k8Tch3h+cV9ZJ8hcD7tYhVNybRAxt8C/rMJuTj5qNGLqmNlL7ChCcdWZp7VPfeFnuG64gzsdzwzf1C9/iEwfpo8LzR+dIHxrlQPLn8N7TPlkWaLiCUR8TXarazP0T7r+Em27/0/fzvP7buafwJ4eQ9Z6/gY8GfAL6rplzckVwKfjYj91QPjYbSf4WrgR8A/RcRXI+IfImL5iDPNt4V2+4NR58rMY8DfAP8D/ID2sbKfZhxbZ3VxH7ls/3Oao9p/RIwB/wa8PzN/2jlvFNky8+eZ+WraZ8q/Afzqmdz/QiLircBsZu4fdZYFvCEzXwtsAt4bEW/snDmCz3Ap7Tbk32fma4CnaLc7RpnpOdUT364B/nX+vFHkiogLgd+m/Y/irwDLaffIG+FsLu6jeobr8Yi4BKD67+xp8rzQ+MoFxmuJiBfRLux3ZuY9TcqWmT+h/YXveuCCiHj2oTCd23lu39X884Ef95D1dF4PXBMRR4Bp2q2Zv21ArmfP/MjMWeDfaf+DOMrP8ChwNDO/VE3fTbvYN+K4ov2P4Fcy83g1PepcbwK+n5k/ysyfAffQPt5GfmwBZ3XPfSntL0RWc/LLhlcNYT+rOLXn/tec+iXOR6vXV3PqlzhfrsYvot3HvLD6833gomre/C9x3lIzUwD/DHxs3vjIsgGvAC6oXr8E+CLwVtpnWZ1fLr2nev1eTv1yaXf1+lWc+uXS92h/sdT35w1McvIL1ZHmon2W97KO1/9F+6xvpMdX9bmtqV7fWOUZ+TFfrTsNvLMJx3u1zlW0r5R5abXeJ4H3jfrYei5fr0WvCX9ofyv+bdq93Q8OYft30e6l/Yz2Wc022j2yL9C+lOrzHQdHALdWWQ4ArY7t/AHtS5kOzzs4W7Qvofou8HHqX371Btr/C/oNTl4e9pZRZgN+nfalht+o1vuLavyy6gfncHXQn1eNv7iaPlzNv6xjWx+s9nuIjqsW+v28ObW4jzRXtf+vc/LS0Q9W4yM9voBXA49Un+NnaBfBJhzzy2mf5Z7fMdaEXB+ifdnoQeBfaBfoRhzz3n5Akgp0NvfcJUmLsLhLUoEs7pJUIIu7JBXI4i5JBbK4S1KBLO6SVKD/B0kKRLJtNg+KAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "clusters_df.cluster_size.hist(log=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write to s3 (JSON)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">date</th>\n",
       "      <th>size</th>\n",
       "      <th>time_delta</th>\n",
       "      <th>newspapers</th>\n",
       "      <th>members</th>\n",
       "      <th>lexical_overlap</th>\n",
       "      <th>cluster_size</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>count</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cluster</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...</td>\n",
       "      <td>76.265823</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1982-07-24</td>\n",
       "      <td>1988-11-22</td>\n",
       "      <td>2</td>\n",
       "      <td>2313 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...</td>\n",
       "      <td>64.285714</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...</td>\n",
       "      <td>82.857143</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...</td>\n",
       "      <td>82.142857</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>1972-02-17</td>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              date             size time_delta newspapers  \\\n",
       "               min        max count                         \n",
       "cluster                                                     \n",
       "0       1982-07-24 1982-07-24     2     0 days    GDL,IMP   \n",
       "1       1982-07-24 1988-11-22     2  2313 days    GDL,IMP   \n",
       "2       1972-02-17 1972-02-17     2     0 days    GDL,IMP   \n",
       "3       1972-02-17 1972-02-17     2     0 days    GDL,IMP   \n",
       "4       1972-02-17 1972-02-17     2     0 days    GDL,IMP   \n",
       "\n",
       "                                                   members lexical_overlap  \\\n",
       "                                                                             \n",
       "cluster                                                                      \n",
       "0        IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...       76.265823   \n",
       "1        IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...       64.285714   \n",
       "2        GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...       82.857143   \n",
       "3        GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...       82.142857   \n",
       "4        GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...       65.000000   \n",
       "\n",
       "        cluster_size  \n",
       "                      \n",
       "cluster               \n",
       "0                  2  \n",
       "1                  2  \n",
       "2                  2  \n",
       "3                  2  \n",
       "4                  2  "
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = clusters_df[[\"time_delta\", \"newspapers\", \"members\", \"lexical_overlap\", \"cluster_size\"]].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters_ddf = dd.from_pandas(temp, chunksize=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dask Index Structure:\n",
       "npartitions=459\n",
       "0         int64\n",
       "1000        ...\n",
       "          ...  \n",
       "458000      ...\n",
       "458568      ...\n",
       "dtype: int64\n",
       "Dask Name: from_pandas, 918 tasks"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_ddf.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>cluster</th>\n",
       "      <th>time_delta</th>\n",
       "      <th>newspapers</th>\n",
       "      <th>members</th>\n",
       "      <th>lexical_overlap</th>\n",
       "      <th>cluster_size</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...</td>\n",
       "      <td>76.265823</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2313 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...</td>\n",
       "      <td>64.285714</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...</td>\n",
       "      <td>82.857143</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...</td>\n",
       "      <td>82.142857</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0 days</td>\n",
       "      <td>GDL,IMP</td>\n",
       "      <td>GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  cluster time_delta newspapers  \\\n",
       "                                  \n",
       "0       0     0 days    GDL,IMP   \n",
       "1       1  2313 days    GDL,IMP   \n",
       "2       2     0 days    GDL,IMP   \n",
       "3       3     0 days    GDL,IMP   \n",
       "4       4     0 days    GDL,IMP   \n",
       "\n",
       "                                             members lexical_overlap  \\\n",
       "                                                                       \n",
       "0  IMP-1982-07-24-a-i0012@0:3756,GDL-1982-07-24-a...       76.265823   \n",
       "1  IMP-1982-07-24-a-i0012@3262:3452,GDL-1988-11-2...       64.285714   \n",
       "2  GDL-1972-02-17-a-i0078@86:1439,IMP-1972-02-17-...       82.857143   \n",
       "3  GDL-1972-02-17-a-i0078@2119:2795,IMP-1972-02-1...       82.142857   \n",
       "4  GDL-1972-02-17-a-i0078@1456:2030,IMP-1972-02-1...       65.000000   \n",
       "\n",
       "  cluster_size  \n",
       "                \n",
       "0            2  \n",
       "1            2  \n",
       "2            2  \n",
       "3            2  \n",
       "4            2  "
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters_ddf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_output = 'test/clusters.json/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "dd.read_json?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dask dataframes has a problem creating json output of a multi-index dataframe\n",
    "# the json it creates it's not valid and cannot be read back into a dask dataframe\n",
    "# so the format below is a workaround\n",
    "output_files = dd.to_json(\n",
    "    clusters_ddf.reset_index(),\n",
    "    url_path=f\"{os.path.join(input_bucket, path_output)}*.json\", \n",
    "    orient='split',\n",
    "    index=False,\n",
    "    storage_options=IMPRESSO_STORAGEOPT\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = db.read_text(\n",
    "    f\"{os.path.join(input_bucket, path_output)}*.json\", \n",
    "    storage_options=IMPRESSO_STORAGEOPT\n",
    ").map(read_json).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_json(string):\n",
    "    try:\n",
    "        return json.loads(string)\n",
    "    except Exception:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = db.read_text(\n",
    "    f\"{os.path.join(input_bucket, path_output)}*.json\", \n",
    "    storage_options=IMPRESSO_STORAGEOPT\n",
    ").map(read_json).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "json.loads?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write output to s3 (CSV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters_ddf = dd.from_pandas(clusters_df, chunksize=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_output = 'test/clusters'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dd.to_csv(\n",
    "    clusters_ddf,\n",
    "    url_path=f\"{os.path.join(input_bucket, path_output)}*.csv\", \n",
    "    storage_options=IMPRESSO_STORAGEOPT\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "dd.to_json?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ddf = dd.read_csv(\n",
    "    urlpath=f\"{os.path.join(input_bucket, path_output)}-*.csv\", \n",
    "    assume_missing=True,\n",
    "    storage_options=IMPRESSO_STORAGEOPT\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask.dataframe.core.DataFrame'>\n",
      "Columns: 8 entries, Unnamed: 0 to lexical_overlap\n",
      "dtypes: object(7), float64(1)"
     ]
    }
   ],
   "source": [
    "test_ddf.info()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "244.875px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}